{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "import lxml.etree as le "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['HTML 教程',\n",
       " 'HTML 简介',\n",
       " 'HTML 编辑器',\n",
       " 'HTML 基础',\n",
       " 'HTML 元素',\n",
       " 'HTML 属性',\n",
       " 'HTML 标题',\n",
       " 'HTML 段落',\n",
       " 'HTML 文本格式化',\n",
       " 'HTML 链接',\n",
       " 'HTML 头部',\n",
       " 'HTML CSS',\n",
       " 'HTML 图像',\n",
       " 'HTML 表格',\n",
       " 'HTML 列表',\n",
       " 'HTML 区块',\n",
       " 'HTML 布局',\n",
       " 'HTML 表单',\n",
       " 'HTML 框架',\n",
       " 'HTML 颜色',\n",
       " 'HTML 颜色名',\n",
       " 'HTML 颜色值',\n",
       " 'HTML 脚本',\n",
       " 'HTML 字符实体',\n",
       " 'HTML URL',\n",
       " 'HTML 速查列表',\n",
       " 'HTML 标签简写及全称',\n",
       " 'HTML 总结',\n",
       " 'XHTML 简介',\n",
       " 'HTML5 教程',\n",
       " 'HTML5 浏览器支持',\n",
       " 'HTML5 新元素',\n",
       " 'HTML5 Canvas',\n",
       " 'HTML5 内联 SVG',\n",
       " 'HTML5 MathML',\n",
       " 'HTML5 拖放',\n",
       " 'HTML5 地理定位',\n",
       " 'HTML5 Video(视频)',\n",
       " 'HTML5 Audio(音频)',\n",
       " 'HTML5 Input 类型',\n",
       " 'HTML5 表单元素',\n",
       " 'HTML5 表单属性',\n",
       " 'HTML5 语义元素',\n",
       " 'HTML5 Web 存储',\n",
       " 'HTML5 Web SQL',\n",
       " 'HTML5 应用程序缓存',\n",
       " 'HTML5 Web Workers',\n",
       " 'HTML5 SSE',\n",
       " 'HTML5 WebSocket',\n",
       " 'HTML5 测验',\n",
       " 'HTML(5) 代码规范',\n",
       " 'HTML 媒体(Media)',\n",
       " 'HTML 插件',\n",
       " 'HTML 音频(Audio)',\n",
       " 'HTML 视频（Video）播放',\n",
       " 'HTML 实例',\n",
       " 'HTML 标签列表(字母排序)',\n",
       " 'HTML 标签列表（功能排序）',\n",
       " 'HTML 属性',\n",
       " 'HTML 事件',\n",
       " 'HTML 画布',\n",
       " 'HTML 音频/视频',\n",
       " 'HTML 有效DOCTYPES',\n",
       " 'HTML 颜色名',\n",
       " 'HTML 拾色器',\n",
       " 'HTML 字符集',\n",
       " 'HTML ASCII',\n",
       " 'HTML ISO-8859-1',\n",
       " 'HTML 符号',\n",
       " 'HTML URL 编码',\n",
       " 'HTML 语言代码',\n",
       " 'HTTP 消息',\n",
       " 'HTTP 方法',\n",
       " '键盘快捷键']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取到地址和利用XPath获取到相应的小结名称的搜索方式\n",
    "url = 'https://www.runoob.com/html/html-tutorial.html'\n",
    "x = '//div[@id=\"leftcolumn\"]/a/text()'\n",
    "\n",
    "#通过request获取到相应地址，搜索相应的小结信息\n",
    "content = requests.get(url).content \n",
    "contentx = le.HTML(content)\n",
    "rets = contentx.xpath(x)\n",
    "\n",
    "#将获取到的小结名称放到列表里面\n",
    "part1 = []\n",
    "for ret in rets:\n",
    "    part1.append(ret.strip())\n",
    "\n",
    "part1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/html/html-tutorial.html',\n",
       " '/html/html-intro.html',\n",
       " '/html/html-editors.html',\n",
       " '/html/html-basic.html',\n",
       " '/html/html-elements.html',\n",
       " '/html/html-attributes.html',\n",
       " '/html/html-headings.html',\n",
       " '/html/html-paragraphs.html',\n",
       " '/html/html-formatting.html',\n",
       " '/html/html-links.html',\n",
       " '/html/html-head.html',\n",
       " '/html/html-css.html',\n",
       " '/html/html-images.html',\n",
       " '/html/html-tables.html',\n",
       " '/html/html-lists.html',\n",
       " '/html/html-blocks.html',\n",
       " '/html/html-layouts.html',\n",
       " '/html/html-forms.html',\n",
       " '/html/html-iframes.html',\n",
       " '/html/html-colors.html',\n",
       " '/html/html-colornames.html',\n",
       " '/html/html-colorvalues.html',\n",
       " '/html/html-scripts.html',\n",
       " '/html/html-entities.html',\n",
       " '/html/html-url.html',\n",
       " '/html/html-quicklist.html',\n",
       " '/html/html-tag-name.html',\n",
       " '/html/html-summary.html',\n",
       " '/html/html-xhtml.html',\n",
       " '/html/html5-intro.html',\n",
       " 'html5-browsers.html',\n",
       " '/html/html5-new-element.html',\n",
       " '/html/html5-canvas.html',\n",
       " '/html/html5-svg.html',\n",
       " 'html5-mathml.html',\n",
       " '/html/html5-draganddrop.html',\n",
       " '/html/html5-geolocation.html',\n",
       " '/html/html5-video.html',\n",
       " '/html/html5-audio.html',\n",
       " '/html/html5-form-input-types.html',\n",
       " '/html/html5-form-elements.html',\n",
       " '/html/html5-form-attributes.html',\n",
       " '/html/html5-semantic-elements.html',\n",
       " '/html/html5-webstorage.html',\n",
       " 'html5-web-sql.html',\n",
       " '/html/html5-app-cache.html',\n",
       " '/html/html5-webworkers.html',\n",
       " '/html/html5-serversentevents.html',\n",
       " '/html/html5-websocket.html',\n",
       " '/quiz/html5-quiz.html',\n",
       " '/html/html5-syntax.html',\n",
       " '/html/html-media.html',\n",
       " '/html/html-object.html',\n",
       " '/html/html-sounds.html',\n",
       " '/html/html-videos.html',\n",
       " '/html/html-examples.html',\n",
       " '/tags/html-reference.html',\n",
       " '/tags/ref-byfunc.html',\n",
       " '/tags/ref-standardattributes.html',\n",
       " '/tags/ref-eventattributes.html',\n",
       " '/tags/ref-canvas.html',\n",
       " '/tags/ref-av-dom.html',\n",
       " '/tags/html-elementsdoctypes.html',\n",
       " '/tags/html-colorname.html',\n",
       " '/tags/html-colorpicker.html',\n",
       " '/charsets/html-charsets.html',\n",
       " '/tags/html-ascii.html',\n",
       " '/tags/ref-entities.html',\n",
       " '/tags/html-symbols.html',\n",
       " '/tags/html-urlencode.html',\n",
       " '/tags/html-language-codes.html',\n",
       " '/tags/html-httpmessages.html',\n",
       " '/tags/html-httpmethods.html',\n",
       " '/tags/html-keyboardshortcuts.html']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取到地址和利用XPath获取到相应的小结链接的搜索方式\n",
    "url_1 = 'https://www.runoob.com/html/html-tutorial.html'\n",
    "x_1 = '//div[@id=\"leftcolumn\"]/a/@href'\n",
    "\n",
    "#通过request获取到相应地址，搜索相应的小结信息的链接地址\n",
    "content = requests.get(url_1).content \n",
    "contentx = le.HTML(content)\n",
    "rets_1 = contentx.xpath(x_1)\n",
    "\n",
    "#将获取到的小结名称放到列表里面\n",
    "part2=[]\n",
    "for ret in rets_1:\n",
    "    part2.append(ret.strip())\n",
    "\n",
    "part2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>小结名称</th>\n",
       "      <th>链接地址</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HTML 教程</td>\n",
       "      <td>/html/html-tutorial.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HTML 简介</td>\n",
       "      <td>/html/html-intro.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HTML 编辑器</td>\n",
       "      <td>/html/html-editors.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HTML 基础</td>\n",
       "      <td>/html/html-basic.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HTML 元素</td>\n",
       "      <td>/html/html-elements.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>HTML URL 编码</td>\n",
       "      <td>/tags/html-urlencode.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>HTML 语言代码</td>\n",
       "      <td>/tags/html-language-codes.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>HTTP 消息</td>\n",
       "      <td>/tags/html-httpmessages.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>HTTP 方法</td>\n",
       "      <td>/tags/html-httpmethods.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>键盘快捷键</td>\n",
       "      <td>/tags/html-keyboardshortcuts.html</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           小结名称                               链接地址\n",
       "0       HTML 教程           /html/html-tutorial.html\n",
       "1       HTML 简介              /html/html-intro.html\n",
       "2      HTML 编辑器            /html/html-editors.html\n",
       "3       HTML 基础              /html/html-basic.html\n",
       "4       HTML 元素           /html/html-elements.html\n",
       "..          ...                                ...\n",
       "69  HTML URL 编码          /tags/html-urlencode.html\n",
       "70    HTML 语言代码     /tags/html-language-codes.html\n",
       "71      HTTP 消息       /tags/html-httpmessages.html\n",
       "72      HTTP 方法        /tags/html-httpmethods.html\n",
       "73        键盘快捷键  /tags/html-keyboardshortcuts.html\n",
       "\n",
       "[74 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#将两个列表中的数据分别整合到Dataframe里面，然后再进行整合\n",
    "df1 = pd.DataFrame(data=part1,columns= ['小结名称'])\n",
    "df2 = pd.DataFrame(data=part2,columns=[ '链接地址'] )\n",
    "df3 = pd.concat([df1,df2],axis=1)\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#存入到excel中\n",
    "df3.to_excel('./XPath作业.xls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
